package com.example.crawler.component;

import com.example.crawler.entity.ProfessionEntity;
import com.example.crawler.mapper.ProfessionMapper;
import com.example.crawler.utils.GsonUtil;
import com.example.crawler.utils.StrUtil;
import com.xiaoleilu.hutool.http.Header;
import com.xiaoleilu.hutool.http.HttpRequest;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.List;
import java.util.Random;

/**
 * com.example.crawler.component
 *
 * @author 白雨浓
 * date: 2017/10/29 15:14
 * content:
 */
@Component
public class ProfessionComponent {
    private static final String USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:56.0) Gecko/20100101 Firefox/56.0";
    private static final String COOKIE = "UM_distinctid=15f620b167d0-0f5b9099fc89a2-72236751-100200-15f620b167e297; CNZZDATA1255363830=60405890-1509173276-null%7C1509201571; Hm_lvt_fd9bf00761167d2e9689ba3d37779881=1509192609,1509192629,1509200287,1509202595; w_phone=d76bpypD9w0f3s1vRktAhAKyDIsXY1Y4uIfTEQHKuOkdWYeBInaZow; w_password=9846nRmCLVnd59BUsj%2FMRXsdQItv91%2FO6Db2l2v%2BJmH4d4JVM3GGIt2Py2U; w_tobe=2f6aw%2FNH5kurzjK0CbGdvTNKlYQGoFQL6TVhyOs; PHPSESSID=2f1roqv8vueei9jg3girvtec30; Hm_lpvt_fd9bf00761167d2e9689ba3d37779881=1509203502; w_liulan=1%2C%E5%8C%97%E4%BA%AC%E5%A4%A7%E5%AD%A6%2Ccollege2%2F5948cbfc5b57c.jpg*359%2C%E5%86%85%E8%92%99%E5%8F%A4%E6%B0%91%E6%97%8F%E5%A4%A7%E5%AD%A6%2Ccollege2%2F5948cc5a31c00.jpg*360%2C%E5%8C%97%E4%BA%AC%E7%AC%AC%E4%BA%8C%E5%A4%96%E5%9B%BD%E8%AF%AD%E5%AD%A6%E9%99%A2%2Ccollege2%2F5948cc5a31c3d.jpg*361%2C%E5%8D%97%E4%BA%AC%E5%AE%A1%E8%AE%A1%E5%A4%A7%E5%AD%A6%2Ccollege2%2F5948cc5a31c7b.jpg";

    @Autowired
    ProfessionMapper mapper;

    public void start() {
        //23
        for (int i = 3; i < 23; i++) {
            handleHtml(i + 1);
        }
    }

    private void handleHtml(int cid) {
        int page = 1;

        while (true) {
            String url = "http://www.gaokaoq.com/job/lists.html?cid=" + cid + "&p=" + page;

            String html = getHtml(url);

            Document doc = Jsoup.parse(html);
            Elements matterElements = doc.select("div.matter h4 a");
            if (matterElements.size() == 0) return;

            for (Element matterElement : matterElements) {
                ProfessionEntity entity = new ProfessionEntity();
                String ph = getHtml("http://www.gaokaoq.com" + matterElement.attr("href"));
                Document pDoc = Jsoup.parse(ph);

                {
                    Elements nameElements = pDoc.select("div.Title h2");
                    if (nameElements.size() == 0) continue;
                    entity.setProfessionName(nameElements.get(0).text());
                }

                {
                    Elements topElements = pDoc.select("div.career_intro tr");
                    /*所属行业*/
                    {
                        Element element = topElements.get(0);
                        String[] ss = element.text().split("：");
                        if (ss.length == 2) {
                            String[] ts = ss[1].split(" - ");
                            entity.setBelong(getString(ts[0]));
                            entity.setTag(getString(ts[1]));
                        }
                    }
                    /*相关职业*/
                    {
                        Element element = topElements.get(1);
                        String[] ss = element.text().split("：");
                        if (ss.length == 2) {
                            String[] ts = ss[1].split("   ");
                            List<String> list = new ArrayList<>();
                            for (String t : ts) {
                                if (!StrUtil.isBlank(getString(t)))
                                    list.add(getString(t));
                            }
                            if (list.size() != 0)
                                entity.setRelated(GsonUtil.toJson(list));
                        }
                    }
                    /*对口专业*/
                    {
                        Element element = topElements.get(2);
                        String[] ss = element.text().split("：");
                        if (ss.length == 2) {
                            String[] ts = ss[1].split("   ");
                            List<String> list = new ArrayList<>();
                            for (String t : ts) {
                                if (!StrUtil.isBlank(getString(t)))
                                    list.add(getString(t));
                            }
                            if (list.size() != 0)
                                entity.setCounterparts(GsonUtil.toJson(list));
                        }
                    }
                    /*content*/
                    {
                        Elements elements = pDoc.select("div.matter");
                        if (elements.size() != 0)
                            entity.setContent(elements.get(0).html());
                    }
                }
                mapper.insert(entity);
                System.out.println("插入职业：" + entity.getProfessionName());
            }
            if (doc.select("a.next").size() == 0) break;
            page++;
        }
    }

    private String getHtml(String url) {
        String ip = "151" + "." + new Random().nextInt(254) + "." + new Random().nextInt(254) + "." + new Random().nextInt(254);
        return HttpRequest.get(url)
                .header(Header.USER_AGENT, USER_AGENT)
                .header(Header.COOKIE, COOKIE)
                .header("X-Forwarded-For", ip)
                .header("Cache-Control", "max-age=0")
                .header("Upgrade-Insecure-Requests", "1")
                .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
                .execute().body();
    }

    private String getString(String s) {
        return s.replace(" ", "");
    }
}
